{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Notes:\n",
    " * SOTA (run 14) on the sample TICs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['/mnt/tess/astronet/checkpoints/extended_23_run_14/1/AstroCNNModel_extended_20210131_212427',\n",
       " '/mnt/tess/astronet/checkpoints/extended_23_run_14/2/AstroCNNModel_extended_20210131_215717',\n",
       " '/mnt/tess/astronet/checkpoints/extended_23_run_14/3/AstroCNNModel_extended_20210131_222957',\n",
       " '/mnt/tess/astronet/checkpoints/extended_23_run_14/4/AstroCNNModel_extended_20210131_230255',\n",
       " '/mnt/tess/astronet/checkpoints/extended_23_run_14/5/AstroCNNModel_extended_20210131_233546',\n",
       " '/mnt/tess/astronet/checkpoints/extended_23_run_14/6/AstroCNNModel_extended_20210201_000826',\n",
       " '/mnt/tess/astronet/checkpoints/extended_23_run_14/7/AstroCNNModel_extended_20210201_004107',\n",
       " '/mnt/tess/astronet/checkpoints/extended_23_run_14/8/AstroCNNModel_extended_20210201_011350',\n",
       " '/mnt/tess/astronet/checkpoints/extended_23_run_14/9/AstroCNNModel_extended_20210201_014655',\n",
       " '/mnt/tess/astronet/checkpoints/extended_23_run_14/10/AstroCNNModel_extended_20210201_021945']"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import os\n",
    "\n",
    "chkpt_root = '/mnt/tess/astronet/checkpoints/extended_23_run_14'\n",
    "data_files = '/mnt/tess/astronet/tfrecords-s33-cam1-sample2/*'\n",
    "tces_file = '/mnt/tess/astronet/tces-s33_cam1_sample.csv'\n",
    "\n",
    "nruns = 10\n",
    "\n",
    "def load_ensemble(chkpt_root, nruns):\n",
    "    checkpts = []\n",
    "    for i in range(nruns):\n",
    "        parent = os.path.join(chkpt_root, str(i + 1))\n",
    "        if not os.path.exists(parent):\n",
    "            break\n",
    "        all_dirs = os.listdir(parent)\n",
    "        if not all_dirs:\n",
    "            break\n",
    "        d, = all_dirs\n",
    "        checkpts.append(os.path.join(parent, d))\n",
    "    return checkpts\n",
    "\n",
    "paths = load_ensemble(chkpt_root, nruns)\n",
    "paths"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Running model 1\n",
      "Binary prediction threshold: 0.2152499407880693 (orientative)\n",
      "757 records\n",
      "Running model 2\n",
      "Binary prediction threshold: 0.2152499407880693 (orientative)\n",
      "757 records\n",
      "Running model 3\n",
      "Binary prediction threshold: 0.2152499407880693 (orientative)\n",
      "757 records\n",
      "Running model 4\n",
      "Binary prediction threshold: 0.2152499407880693 (orientative)\n",
      "757 records\n",
      "Running model 5\n",
      "Binary prediction threshold: 0.2152499407880693 (orientative)\n",
      "757 records\n",
      "Running model 6\n",
      "Binary prediction threshold: 0.2152499407880693 (orientative)\n",
      "757 records\n",
      "Running model 7\n",
      "Binary prediction threshold: 0.2152499407880693 (orientative)\n",
      "757 records\n",
      "Running model 8\n",
      "Binary prediction threshold: 0.2152499407880693 (orientative)\n",
      "757 records\n",
      "Running model 9\n",
      "Binary prediction threshold: 0.2152499407880693 (orientative)\n",
      "757 records\n",
      "Running model 10\n",
      "Binary prediction threshold: 0.2152499407880693 (orientative)\n",
      "757 records\n"
     ]
    }
   ],
   "source": [
    "import getpass\n",
    "import os\n",
    "from astronet import predict\n",
    "import tensorflow as tf\n",
    "\n",
    "\n",
    "def run_predictions(path):\n",
    "    predict.FLAGS = predict.parser.parse_args([\n",
    "      '--model_dir', path,\n",
    "      '--data_files', data_files,\n",
    "      '--output_file', '',\n",
    "    ])\n",
    "\n",
    "    return predict.predict()\n",
    "\n",
    "\n",
    "paths = load_ensemble(chkpt_root, nruns)\n",
    "ensemble_preds = []\n",
    "config = None\n",
    "for i, path in enumerate(paths):\n",
    "    print(f'Running model {i + 1}')\n",
    "    preds, config = run_predictions(path)\n",
    "    ensemble_preds.append(preds.set_index('tic_id'))\n",
    "    print()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "labels = ['disp_E', 'disp_N', 'disp_J', 'disp_S', 'disp_B']\n",
    "\n",
    "col_e = labels.index('disp_E')\n",
    "# thresh = config.hparams.prediction_threshold\n",
    "# thresh = 0.030485098838860747  # From the validation numbers - maximum thrershold for 100% recall\n",
    "thresh = 0.31245827674871207  # Relaxed to match Liang's precision value"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "\n",
    "agg_preds = {}\n",
    "\n",
    "for preds in ensemble_preds:\n",
    "    for tic_id in preds.index:\n",
    "        if tic_id not in agg_preds:\n",
    "            agg_preds[tic_id] = []\n",
    "\n",
    "        row = preds[preds.index == tic_id]\n",
    "        pred_v = row.values[0]\n",
    "        if len(row.values) > 1:\n",
    "            print(f'Warning: duplicate predictions for {tic_id}')\n",
    "        if pred_v[col_e] >= thresh:\n",
    "            agg_preds[tic_id].append('disp_E')\n",
    "        else:\n",
    "            masked_v = [v if i != col_e else 0 for i, v in enumerate(pred_v)]\n",
    "            agg_preds[tic_id].append(preds.columns[np.argmax(masked_v)])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "final_preds = []\n",
    "for tic_id in list(agg_preds.keys()):\n",
    "    counts = {l: 0 for l in labels}\n",
    "    for e in agg_preds[tic_id]:\n",
    "        counts[e] += 1\n",
    "    maxcount = max(counts.values())\n",
    "    counts.update({\n",
    "        'tic_id': tic_id,\n",
    "        'maxcount': maxcount,\n",
    "    })\n",
    "    final_preds.append(counts)\n",
    "\n",
    "final_preds = pd.DataFrame(final_preds).set_index('tic_id')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>disp_E</th>\n",
       "      <th>disp_N</th>\n",
       "      <th>disp_J</th>\n",
       "      <th>disp_S</th>\n",
       "      <th>disp_B</th>\n",
       "      <th>maxcount</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>tic_id</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>11690157</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>10</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>232479493</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>10</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>220338896</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>10</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>265446651</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>10</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>251192557</th>\n",
       "      <td>10</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>10</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "           disp_E  disp_N  disp_J  disp_S  disp_B  maxcount\n",
       "tic_id                                                     \n",
       "11690157        0       0      10       0       0        10\n",
       "232479493       0       0      10       0       0        10\n",
       "220338896       0       0      10       0       0        10\n",
       "265446651       0       0      10       0       0        10\n",
       "251192557      10       0       0       0       0        10"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "final_preds.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>disp_E</th>\n",
       "      <th>disp_N</th>\n",
       "      <th>disp_J</th>\n",
       "      <th>disp_S</th>\n",
       "      <th>disp_B</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>tic_id</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>263337671</th>\n",
       "      <td>0.002818</td>\n",
       "      <td>0.020683</td>\n",
       "      <td>0.904438</td>\n",
       "      <td>0.263909</td>\n",
       "      <td>0.003210</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>263337671</th>\n",
       "      <td>0.014255</td>\n",
       "      <td>0.030863</td>\n",
       "      <td>0.666594</td>\n",
       "      <td>0.534820</td>\n",
       "      <td>0.001861</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>263337671</th>\n",
       "      <td>0.002179</td>\n",
       "      <td>0.013056</td>\n",
       "      <td>0.929887</td>\n",
       "      <td>0.245605</td>\n",
       "      <td>0.003417</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>263337671</th>\n",
       "      <td>0.003771</td>\n",
       "      <td>0.027332</td>\n",
       "      <td>0.897167</td>\n",
       "      <td>0.277175</td>\n",
       "      <td>0.007390</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>263337671</th>\n",
       "      <td>0.005565</td>\n",
       "      <td>0.028687</td>\n",
       "      <td>0.923941</td>\n",
       "      <td>0.144937</td>\n",
       "      <td>0.002416</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>263337671</th>\n",
       "      <td>0.002657</td>\n",
       "      <td>0.010396</td>\n",
       "      <td>0.937068</td>\n",
       "      <td>0.210296</td>\n",
       "      <td>0.001473</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>263337671</th>\n",
       "      <td>0.002572</td>\n",
       "      <td>0.023736</td>\n",
       "      <td>0.930956</td>\n",
       "      <td>0.195460</td>\n",
       "      <td>0.001471</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>263337671</th>\n",
       "      <td>0.024971</td>\n",
       "      <td>0.016409</td>\n",
       "      <td>0.590290</td>\n",
       "      <td>0.566074</td>\n",
       "      <td>0.002671</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>263337671</th>\n",
       "      <td>0.002346</td>\n",
       "      <td>0.017249</td>\n",
       "      <td>0.905677</td>\n",
       "      <td>0.290867</td>\n",
       "      <td>0.003571</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>263337671</th>\n",
       "      <td>0.003239</td>\n",
       "      <td>0.037706</td>\n",
       "      <td>0.961283</td>\n",
       "      <td>0.079129</td>\n",
       "      <td>0.002276</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "             disp_E    disp_N    disp_J    disp_S    disp_B\n",
       "tic_id                                                     \n",
       "263337671  0.002818  0.020683  0.904438  0.263909  0.003210\n",
       "263337671  0.014255  0.030863  0.666594  0.534820  0.001861\n",
       "263337671  0.002179  0.013056  0.929887  0.245605  0.003417\n",
       "263337671  0.003771  0.027332  0.897167  0.277175  0.007390\n",
       "263337671  0.005565  0.028687  0.923941  0.144937  0.002416\n",
       "263337671  0.002657  0.010396  0.937068  0.210296  0.001473\n",
       "263337671  0.002572  0.023736  0.930956  0.195460  0.001471\n",
       "263337671  0.024971  0.016409  0.590290  0.566074  0.002671\n",
       "263337671  0.002346  0.017249  0.905677  0.290867  0.003571\n",
       "263337671  0.003239  0.037706  0.961283  0.079129  0.002276"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def compare(ensemble_preds, filter):\n",
    "    result = ensemble_preds[0][filter]\n",
    "    for preds in ensemble_preds[1:]:\n",
    "        result = result.append(preds[filter])\n",
    "    return result\n",
    "\n",
    "compare(ensemble_preds, preds.index == 263337671)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "def final_pred(row):\n",
    "#     if (row['Distinct'] > 1\n",
    "#         and (\n",
    "#             not isinstance(row['Decision'], str)\n",
    "#             and (\n",
    "#                 (row['av'] in ('E', 'S'))\n",
    "#                 or (row['md'] in ('E', 'S'))\n",
    "#                 or (row['ch'] in ('E', 'S'))\n",
    "#                 or (row['as'] in ('E', 'S'))\n",
    "#                 or (row['mk'] in ('E', 'S'))\n",
    "#                 or (row['et'] in ('E', 'S'))\n",
    "#             )\n",
    "#         )\n",
    "#        ):\n",
    "#         return '?'\n",
    "    \n",
    "    if (row['disp_E'] > 0):\n",
    "        return 'E'\n",
    "    else:\n",
    "        maxpred = 'disp_E'\n",
    "        for c in ['disp_N', 'disp_J', 'disp_S', 'disp_B']:\n",
    "            if row[c] > row[maxpred]:\n",
    "                maxpred = c\n",
    "        return maxpred[5]\n",
    "\n",
    "agg_preds = pd.read_csv('~/Astronet-Triage/Labels - extended mission test.csv', header=0, low_memory=False)\n",
    "agg_preds = agg_preds.set_index('TIC ID')\n",
    "agg_preds = final_preds.join(agg_preds)\n",
    "agg_preds['final'] = agg_preds.apply(final_pred, axis=1)\n",
    "agg_preds = agg_preds[['final']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>final</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>tic_id</th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Empty DataFrame\n",
       "Columns: [final]\n",
       "Index: []"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "agg_preds[agg_preds['final'] == '?']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "agg_preds.to_csv('~/Astronet-Triage/tces-s33_cam1_sample-preds.csv')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
